/*
 * PBBxml.java
 *
 * Created on June 6, 2007, 12:10 PM
 * This class is used to convert the xml output to simple format.
 *
 *
 *Copyright 2007-2008 Jon W Huss III

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 *
 */

package proteinboxbot;

import java.io.*;
import org.xml.sax.*;
import org.xml.sax.helpers.DefaultHandler;
import javax.xml.parsers.*;
import proteinboxbot.*;
import java.util.*;
import org.w3c.dom.*;
import java.net.*;



/**
 *
 * @author Jon Huss
 */
public class PBBxml extends DefaultHandler{
    
    public static final String ENTRY_SEPARATOR = ";__PBB_SPLIT_14554_XMGIT_8Y88N__;";
    
    private Writer out;  // basically will just be the screen for now.
    private String inFile;
    private String outFile;
    private String outDirectory;
    private String outSubDir1; //Sloppy - need to change to a vector
    private String outSubDir2;
    
    private Hashtable<String, Vector> dataHash;
    private int speciesCount;
    private Vector<Hashtable<String,Vector>> speciesData;
    
    private String spacer; //going to try and get a little tabbing in the output.
    
    private Document importDoc;
    //private Document goDoc;
    
    private PBBLogging log;
    
    
    /** Creates a new instance of PBBxml */
    public PBBxml(String outDir,String in, PBBLogging l) {
        inFile = in;
        //this will change later but for now:
        //outDirectory = "c:/temp/;
        outDirectory = outDir;
        
        spacer = "";
        speciesCount = -1;
        speciesData = new Vector<Hashtable<String,Vector>>();
        //dataHash = new Hashtable<String, Vector>();
        out = null;
        outSubDir1 = "";
        outSubDir2 = "";
        log = l;
        
    }
    
    public Writer setOutFile(String outF) {
        //sets up the writer to dump into a file.
        
        //junk test code:
        //File junkFile = new File("c:/temp dir/test again");
        //junkFile.mkdirs();
        
        
        File outputFile;
        try{
            if (outF != null) {
                //System.out.println("creating new file object");
                outputFile = new File(outF);
            } else {
                return null;
            }
            
            if(outputFile.exists()) {
                //System.out.println("deleting file");
                outputFile.delete();
            }
            
            File tmpFile = new File(outputFile.getParent());
            //System.out.println("creating new directories:" + tmpFile.getAbsolutePath());
            tmpFile.mkdirs();
            //System.out.println("creating new file");
            
            outputFile.createNewFile();
            
            
            if (outputFile.canWrite()) {
                return new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(outputFile)));
            } else{
                System.out.println("Cannot Write to output.");
                return null;
            }
        } catch(Exception e) {
            System.out.println("Error while building output.");
            e.printStackTrace();
            return null;
        }
    }
    
    public void xmlParse() {
        SAXParserFactory factory = SAXParserFactory.newInstance();
        try {
            // Set up output stream
            out = new OutputStreamWriter(System.out); //will want to change this later XXX
            
            // Parse the input
            SAXParser saxParser = factory.newSAXParser();
            saxParser.parse( new File(inFile), this);
            
        } catch (Throwable t) {
            t.printStackTrace();
        }
    }
    
    public static Hashtable<String,String> xmlGoParse(String in, PBBLogging slog){
        //This function is specifically for the parsing of "Go" Terms from
        //obo-xml format.  Returns a data dictionary of the terms.
        System.out.println("Parsing GO File.  This may take a few moments.");
        //System.out.println("Starting DOM GO parse.");
        DocumentBuilderFactory factory =
                DocumentBuilderFactory.newInstance();
        //factory.setValidating(true);
        //factory.setNamespaceAware(true);
        Hashtable<String,String> values = null;
        
        Document goDoc;
        
        try {
            slog.addEntry("System","XML_GO","Starting GO term Parsing.",PBBLogging.logs.SY);
            DocumentBuilder builder = factory.newDocumentBuilder();
            goDoc = builder.parse( new File(in) );
            values = PBBxml.doDomGO(goDoc);
            //Output a parsed version of the go file before we are done.
            String out = in + ".pbb";
            File goOut = new File(out);
            if (goOut.exists() == true ){
                goOut.delete();
            }
            slog.addEntry("System","XML_GO","Outputting parsed Go Terms to:" + out,PBBLogging.logs.SY);
            
            goOut.createNewFile();
            BufferedWriter bw = new BufferedWriter(new FileWriter(goOut));
            Set keys = values.keySet();
            Iterator<String> hti = keys.iterator();
            while (hti.hasNext()){
                String key = hti.next();
                String val = values.get(key);
                bw.write(key + ";" + val);
                bw.newLine();
            }
            bw.flush();
            bw.close();
            slog.addEntry("System","XML_GO","Finished GO Term Parsing.",PBBLogging.logs.SY);
            
        } catch (SAXException sxe) {
            // Error generated during parsing)
            Exception  x = sxe;
            if (sxe.getException() != null)
                x = sxe.getException();
            x.printStackTrace();
            
        } catch (ParserConfigurationException pce) {
            // Parser with specified options can't be built
            pce.printStackTrace();
            
        } catch (IOException ioe) {
            // I/O error
            ioe.printStackTrace();
        }
        
        System.out.println("Finished Parsing GO File.");
        
        
        //System.out.println("Finished DOM GO Parse.");
        //return files;
        
        return values;
    }
    
    public static Hashtable<String,String> doDomGO(Document goDoc) {
        //Parsing for the GO terms
        Hashtable<String,String> values = new Hashtable<String,String>();
        
        Element root = goDoc.getDocumentElement();
        
        NodeList termList = root.getElementsByTagName("term");
        String goID = "";
        String goDesc = "";
        
        
        for (int i = 0; i < termList.getLength(); i++){
            
            Node termElement;
            termElement = termList.item(i);
            NodeList children = termElement.getChildNodes();
            Vector<String> goIDs = new Vector<String>();
            for (int j = 0; j < children.getLength(); j++){
                //possibility of more than one id mapping
                if ("id".compareTo(children.item(j).getNodeName())==0 ){
                    //System.out.println(children.item(j).getNodeName()); //debug XXX
                    goIDs.add(children.item(j).getTextContent().trim());
                    //System.out.println(goID); //debug XXX
                }
                if ("alt_id".compareTo(children.item(j).getNodeName())==0 ){
                    goIDs.add(children.item(j).getTextContent().trim());
                }
                if ("name".compareTo(children.item(j).getNodeName())==0 ){
                    //System.out.println(children.item(j).getNodeName()); //debug XXX
                    goDesc = children.item(j).getTextContent().trim();
                    //System.out.println(goDesc); //debug XXX
                }
            }
            if (goIDs != null){
                for (int j = 0; j< goIDs.size(); j++){
                    values.put(goIDs.get(j),goDesc);
                }
            }
            
            
        }
        
        return values;
    }
    
    public Vector<String> htmlParse(InputStream inPage) {
        Vector<String> data = null;
        log.addEntry("System","XML_HTML","Starting Web Parse",PBBLogging.logs.SY);
        
        //System.out.println("Starting Web parse.");
        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
        Document importPage;
        try {
            DocumentBuilder builder = factory.newDocumentBuilder();
            importPage = builder.parse( inPage );
            //System.out.println("Retrieving Data...");
            data = this.htmlDataMine(importPage);
            
            log.addEntry("System","XML_HTML","Finished Parsing.",PBBLogging.logs.SY);
            
        } catch (Exception e) {
            e.printStackTrace();
        }
        
        return data;
        
    }
    
    public Vector<String> htmlDataMine(Document doc) throws Exception {
        Vector<String> vals = null;
        
        Element root = doc.getDocumentElement();
        NodeList inputList = root.getElementsByTagName("input");
        
        vals = new Vector<String>();
        
        for (int i = 0; i < inputList.getLength(); i++){
            Element tmpElement = (Element)inputList.item(i);
            
            //System.out.println("Input #" + i +": " + tmpElement.getAttribute("name") + " = " + tmpElement.getAttribute("value"));
            if ("hidden".compareToIgnoreCase(tmpElement.getAttribute("type")) == 0){
                if ("".compareToIgnoreCase(tmpElement.getAttribute("value").trim()) !=0){
                    vals.add(URLEncoder.encode(tmpElement.getAttribute("name"), "UTF-8") + "=" + URLEncoder.encode(tmpElement.getAttribute("value"), "UTF-8"));
                }
            }
            
        }
        
        return vals;
    }
    
    private String[] xmlCiteBuilderDocParser(Document doc, int maxReturn){
        //The workhorse of the citation builder.
        
        String[] citations = null;
        
//        {{cite journal
//        |author=Koeffler HP
//                 |title=Peroxisome proliferator-activated receptor gamma and cancers
//                 |journal=Clin. Cancer Res.
//                 |volume=9
//                 |issue=1
//                 |pages=1-9
//                 |year=2003
//                 |pmid=12538445
//                 |doi=
//        }}
        
        
        Element supRoot = doc.getDocumentElement();
        //now have the root node.
        NodeList artList = supRoot.getElementsByTagName("PubmedArticle");
        //Just abort if we get no nodes..
        if(artList.getLength() < 1) {
            citations = new String[0];
            //citations[0] = "";
            return citations;
        }
        if (maxReturn > artList.getLength()) {
            maxReturn = artList.getLength();
        }
        citations = new String[maxReturn];
        for (int i = 0; i < maxReturn; i++){
            
            Element root = (Element)artList.item(i);
            
            String author = "";
            String title = "";
            String journal = "";
            String volume = "";
            String issue = "";
            String year = "";
            String pmid = "";
            String doi = "";
            String pages = "";
            
            
            //Start gathering up data...
            NodeList tmpList = root.getElementsByTagName("PMID");
            Element tmpElement = (Element)tmpList.item(0);
            if (tmpElement != null) {
                pmid = tmpElement.getTextContent();
            }
            
            tmpList = root.getElementsByTagName("DateCompleted");
            if (tmpList.getLength() > 0){
                tmpElement = (Element)tmpList.item(0);
                if (tmpElement != null) {
                    tmpList = tmpElement.getElementsByTagName("Year");
                    tmpElement = (Element)tmpList.item(0);
                    
                    if (tmpElement != null) {
                        year = tmpElement.getTextContent();
                    }
                }
            }
            
            
            //We have added in a convention to replace authors past the third with
            //et al if there were five or more authors..
            tmpList = root.getElementsByTagName("Author");
            int maxAuth;
            if (tmpList.getLength() > 4){
                maxAuth = 3;
            } else {
                maxAuth = tmpList.getLength();
            }
            for (int j = 0; j < maxAuth; j++){
                
                String lname = "";
                String initials = "";
                tmpElement = (Element)tmpList.item(j);
                NodeList tmpListSub = tmpElement.getElementsByTagName("Initials");
                Element tmpElementSub = (Element)tmpListSub.item(0);
                if (tmpElementSub != null){
                    initials = tmpElementSub.getTextContent();
                }
                tmpListSub = tmpElement.getElementsByTagName("LastName");
                tmpElementSub = (Element)tmpListSub.item(0);
                if (tmpElementSub != null){
                    lname = tmpElementSub.getTextContent();
                }
                
                author += ", " + lname + " " + initials;
            }
            
            if (author.length() > 1){
                author = author.substring(1).trim();
            }
            
            if (maxAuth < tmpList.getLength()){
                //Just need to add in et al at this point...
                author += ", ''et al.''";
            }
            
            //First we want to see if there is an abbreviation for the journal.. otherwise, use the Title.
            tmpList = root.getElementsByTagName("ISOAbbreviation");
            tmpElement = (Element)tmpList.item(0);
            if (tmpElement != null) {
                journal = tmpElement.getTextContent();
            }
            
            if (journal.compareTo("") == 0){
                tmpList = root.getElementsByTagName("Title");
                tmpElement = (Element)tmpList.item(0);
                if (tmpElement != null) {
                    journal = tmpElement.getTextContent();
                }
            }
            
            tmpList = root.getElementsByTagName("ArticleTitle");
            tmpElement = (Element)tmpList.item(0);
            if (tmpElement != null) {
                title = tmpElement.getTextContent();
            }
            
            tmpList = root.getElementsByTagName("Volume");
            tmpElement = (Element)tmpList.item(0);
            if (tmpElement != null) {
                volume = tmpElement.getTextContent();
            }
            
            tmpList = root.getElementsByTagName("Issue");
            tmpElement = (Element)tmpList.item(0);
            if (tmpElement != null) {
                issue = tmpElement.getTextContent();
            }
            
            tmpList = root.getElementsByTagName("MedlinePgn");
            tmpElement = (Element)tmpList.item(0);
            if (tmpElement != null) {
                pages = tmpElement.getTextContent();
            }
            
            DOI_SEARCH: {
                tmpList = root.getElementsByTagName("ArticleId");
                for (int j = 0; j < tmpList.getLength(); j++){
                    tmpElement = (Element)tmpList.item(j);
                    if (tmpElement.getAttribute("IdType").compareTo("doi") == 0) {
                        doi = tmpElement.getTextContent();
                        break DOI_SEARCH;
                        
                        //Found the doi content.
                        //tmpList = tmpElement.getChildNodes();
                        //for (int k = 0; k< tmpList.getLength(); k++){
                        //    //need to find the text node..
                        //    tmpElement = (Element)tmpList.item(k);
                        //    if (tmpElement.getNodeType() == Element.TEXT_NODE){
                        //        doi = tmpElement.getTextContent();
                        //        break DOI_SEARCH;
                        //    }
                        //}
                        
                    }
                }
            }// END DOI_SEARCH
            
            
            
            String data = "";
            data = "{{cite journal " +
                    " | author=" + author +
                    " |title=" + title +
                    " |journal=" + journal +
                    " |volume=" + volume +
                    " |issue= " + issue +
                    " |pages= " + pages +
                    " |year= " + year +
                    " |pmid= " + pmid +
                    " |doi= " + doi + " }}";
            
            citations[i] = data;
        }
        return citations;
    }
    
    public String[] xmlCiteBuilder(File xmlDataFile, int maxReturn) {
        File inF = xmlDataFile;
        if (inF.canRead()) {
            String data;
            try {
                //Gather File info..
                String line;
                BufferedReader br = new BufferedReader(new FileReader(inF));
                data = "";
                while ((line = br.readLine()) != null){
                    data += line;
                }
                return xmlCiteBuilder(data, maxReturn);
            } catch (FileNotFoundException ex) {
                ex.printStackTrace();
            } catch (IOException ex) {
                ex.printStackTrace();
            }
            
        }
        return null;
    }
    
    
    public String[] xmlCiteBuilder(String xmlData, int maxReturn) {
        
        System.out.println("Starting DOM parse.");
        DocumentBuilderFactory factory =
                DocumentBuilderFactory.newInstance();
        //factory.setValidating(true);
        //factory.setNamespaceAware(true);
        String citation[] = null;
        try {
            Document in;
            
            DocumentBuilder builder = factory.newDocumentBuilder();
            
            in = builder.parse(new ByteArrayInputStream(xmlData.getBytes("UTF-8")));
            citation = this.xmlCiteBuilderDocParser(in, maxReturn);
            
        } catch (SAXException sxe) {
            // Error generated during parsing)
            Exception  x = sxe;
            if (sxe.getException() != null)
                x = sxe.getException();
            x.printStackTrace();
            
        } catch (ParserConfigurationException pce) {
            // Parser with specified options can't be built
            pce.printStackTrace();
            
        } catch (IOException ioe) {
            // I/O error
            ioe.printStackTrace();
            Throwable tmp;
            while((tmp = ioe.getCause()) != null){
                tmp.getLocalizedMessage();
            }
            
        }
        
        System.out.println("Finished DOM Parse.");
        return citation;
        
        
        
    }
    
    public String[] xmlGetQKAWEHelper(Document in){
        String[] data = new String[2];
        data[0] = ""; //Query Key
        data[1] = ""; //WebEnv
        
        Element root = in.getDocumentElement();
        //now have the root node.
        NodeList dbList = root.getElementsByTagName("LinkSetDbHistory");
        //Just abort if we get no nodes..
        if(dbList.getLength() < 1) {
            return data;
        }
        for (int i =0; i< dbList.getLength(); i++){
            Element tempElement = (Element)dbList.item(i);
            NodeList tmpList = tempElement.getElementsByTagName("LinkName");
            if (tmpList.getLength() < 1) {
                continue;
            }
            Element tElm = (Element)tmpList.item(0);
            if (tElm.getTextContent().trim().compareTo("gene_pubmed") == 0) {
                //found the right one.
                tmpList = tempElement.getElementsByTagName("QueryKey");
                if (tmpList.getLength() < 1) {
                    continue;
                }
                tElm = (Element)tmpList.item(0);
                data[0] = tElm.getTextContent().trim();
                //Got what we need so Break.
                break;
            }
        }
        
        dbList = root.getElementsByTagName("WebEnv");
        if(dbList.getLength() < 1) {
            return data;
        }
        //Got the WebEnv.
        Element tmpElement = (Element)dbList.item(0);
        data[1] = tmpElement.getTextContent().trim();
        
        return data;
    }
    
    public String[] xmlGetQueryKeyAndWebEnv(String xmlData) {
        
        //Returns a string array of length 2
        //the first string is the query key
        //the second is the WebEnv
        
        System.out.println("Starting DOM parse.");
        DocumentBuilderFactory factory =
                DocumentBuilderFactory.newInstance();
        //factory.setValidating(true);
        //factory.setNamespaceAware(true);
        String citation[] = null;
        try {
            Document in;
            
            DocumentBuilder builder = factory.newDocumentBuilder();
            
            in = builder.parse(new ByteArrayInputStream(xmlData.getBytes("UTF-16")));
            citation = this.xmlGetQKAWEHelper(in);
            
        } catch (SAXException sxe) {
            // Error generated during parsing)
            Exception  x = sxe;
            if (sxe.getException() != null)
                x = sxe.getException();
            x.printStackTrace();
            
        } catch (ParserConfigurationException pce) {
            // Parser with specified options can't be built
            pce.printStackTrace();
            
        } catch (IOException ioe) {
            // I/O error
            ioe.printStackTrace();
            Throwable tmp;
            while((tmp = ioe.getCause()) != null){
                tmp.getLocalizedMessage();
            }
            
        }
        
        System.out.println("Finished DOM Parse.");
        return citation;
        
        
        
    }
    
    public String[] xmlDomParse(PBBxmlDecoder[] dec) {
        System.out.println("Starting DOM parse.");
        DocumentBuilderFactory factory =
                DocumentBuilderFactory.newInstance();
        //factory.setValidating(true);
        //factory.setNamespaceAware(true);
        String[] files = null;
        try {
            DocumentBuilder builder = factory.newDocumentBuilder();
            //importDoc - builder.parse( new )
            importDoc = builder.parse( new File(inFile) );
            files = this.doDomOutput(dec);
            
        } catch (SAXException sxe) {
            // Error generated during parsing)
            Exception  x = sxe;
            if (sxe.getException() != null)
                x = sxe.getException();
            x.printStackTrace();
            
        } catch (ParserConfigurationException pce) {
            // Parser with specified options can't be built
            pce.printStackTrace();
            
        } catch (IOException ioe) {
            // I/O error
            ioe.printStackTrace();
        }
        
        System.out.println("Finished DOM Parse.");
        return files;
        
    }
    
    public String[] doDomOutput(PBBxmlDecoder[] decoder) throws SAXException {
        //Basic idea is to see if we can generate an output file in simple format with all necessary info for the PBF
        //should be able to extract the values needed. We will do this with a value list.
        
        //Returns an array of strings that are the filenames of the output files.
        //No filenames are available then null is returned.
        String[] fileNames = null;
        if (decoder == null){
            throw new SAXException("Null Decoder");
        }
        if (decoder.length < 1 ){
            throw new SAXException("Null Decoder");
        }
        /*
        if (out != null){
            try {
                nl();
                out.flush();
            } catch (IOException e) {
                throw new SAXException("I/O error", e);
            }
        }
        System.out.println("Setting new output for DOM dump.");
        out = this.setOutFile("c:/temp/domdump.txt");
        if (out == null) {
            out = new OutputStreamWriter(System.out);
        }
         */
        //We now have the output setup - lets start probing for values.
        Element tmpElement;
        //need to cut up the document into distinct homologies.
        Element root = importDoc.getDocumentElement();
        String tempStr = "";
        
        
        NodeList homologyList = root.getElementsByTagName("Orthology_ID");
        //Now have a list of the individual genes/proteins
        
        Vector<String> homologies = new Vector<String>();
        //intro to file.
        //emit("# **** Output Begin ****"); nl();
        for (int h = 0; h < homologyList.getLength(); h++){
            
            Element homologyElement;
            homologyElement = (Element)homologyList.item(h);
            String tmpH = homologyElement.getAttribute("value");
            if(!homologies.contains(tmpH)){
                homologies.add(tmpH);
            }
        }
        
        String temp1= "";
        String temp2= "";
        String accessDate = "";
        
        //The first subdirectory will be the root info.
        temp1 = root.getNodeName();
        if (temp1.compareTo("gene_report") == 0) {
            //ok, looks like the root node is a little higher than anticipated..
            //we need to grab the child node called BioGPS_Report
            NodeList tempNodeList = root.getElementsByTagName("BioGPS_Report");
            if (tempNodeList.getLength() > 0) {
                root = (Element)tempNodeList.item(0); //Hope this works..
            }
            
            
        }
        accessDate = root.getAttribute("DateCreated").trim();
        temp2 = accessDate.replace(':','.');
        String accessYear = accessDate.substring(6,10);
        String accessMoDa = accessDate.substring(0,5);
        accessDate = accessYear + "-" + accessMoDa;
        outSubDir1 = temp1 + "-" + temp2 + "/";
        
        if (homologies.size() > 0){
            fileNames = new String[homologies.size()];
        }
        
        
        for (int i = 0; i <homologies.size(); i++){
            //want to build a new file for every homology:
            String baseAddress = "Orthology_ID{{att}}value{{val}}" + homologies.get(i);
            
            //Gonna need to allow more customization here XXX
            outFile = this.seekValue(root,(baseAddress + " " + "GeneReport{{att}}TaxonomyName{{val}}Human Gene_Accession Symbol{{att}}value{{val}}??").split("\\s+"),0, ", ");
            if (outFile.length() > 0){
                outFile = outFile.substring(1).trim(); //gets rid of the lead comma
            }
            
            outFile = "PBB_Protein-"+ outFile + "-" + homologies.get(i) +".simple";
            //need to start a new output file stream.
            String outTempName = outDirectory+outSubDir1+outSubDir2+outFile;
            fileNames[i] = outTempName;
            //Need to update the SpeciesCount and start a new dataset.
            //    speciesCount++;
            //speciesData.add(new Hashtable<String, Vector>());
            try {
                //nl();
                if (out == null) {
                    out = new OutputStreamWriter(System.out);
                }
                out.flush();
            } catch (IOException e) {
                throw new SAXException("I/O error", e);
            }
            log.addEntry("System","XML","Setting xml output Directory:" + outTempName,PBBLogging.logs.SY);
            
            //System.out.println("Setting new output Directory: "+ outTempName);
            out = this.setOutFile(outTempName);
            
            //Should have a unique list of Homologies now.
            
            
            //Now we need to pluck out values.
            emit("#"); nl(); emit("#"); nl(); emit("# **** Begin Homology: "+ homologies.get(i)+ " ****"); nl();
            //Dump out the access date:
            emit("AccessDate" + this.ENTRY_SEPARATOR + accessDate); nl();
            
            //Now we will go through the address list.. for now a check to see if things are working.
            
            for (int j = 0; j <decoder.length;j++){
                String name = decoder[j].getName();
                String theAddress = decoder[j].getAddress();
                String val = this.seekValue(root,(baseAddress + " " + theAddress).split("\\s+"),0,this.ENTRY_SEPARATOR);
                val = val.trim();
                emit("# "+ name +": "+ (baseAddress + " " + theAddress)); nl();
                if (decoder[j].getUseReplace()){
                    if (name.compareTo("") == 0){
                        if (val.length() > 0){
                            val = val.substring(this.ENTRY_SEPARATOR.length()).trim(); //gets rid of the lead separator
                        }
                        //we have an interesting case, odds are that the name is included in the val.
                        //could be more than one, so we are gonna split this up.
                        String[] valSplit = val.split(this.ENTRY_SEPARATOR);
                        for (int k = 0; k < valSplit.length; k++){
                            //Since we have standardized the split - we just need to know what to change into the split code.
                            //I.e. we only need to know the From, and not the To.
                            //val = valSplit[k].replace(decoder[j].getReplaceCodeFrom(),decoder[j].getReplaceCodeTo()).trim();
                            val = valSplit[k].replace(decoder[j].getReplaceCodeFrom(),this.ENTRY_SEPARATOR).trim();
                            emit(val); nl();
                        }
                    } else {
                        
                        val = val.replace(decoder[j].getReplaceCodeFrom(),decoder[j].getReplaceCodeTo());
                        emit(name + val); nl();
                    }
                    
                } else {
                    
                    
                    emit(name + val); nl();
                }
            }
        }
        return fileNames;
    }
    
    private String seekValue(Element root, String[] address, int addressIndex, String valueSplit){
        // the address will consist of values to seach down for (separated by white space) and they can contain a subattribute with {{att}} and {{val}}
        // if a "??" follows the VAL, then we assume that this is the value to be returned.
        //Note that this function will not look at the current root, only its children.
        
        //Note that this is a semi-recursive algorithm.
        //addressIndex is the index for which part of the address to look at.
        
        String[] addressParts;
        String addressName = "";
        
        String addressAtt = "";
        String addressAttVal = "";
        //boolean getAttVal = false;
        boolean checkAtt = false;
        String returnValue = "";
        
        if (addressIndex >= address.length){
            log.addEntry("System","XML_SEEK_ERROR","Bad Index while Seeking.",PBBLogging.logs.SY);
            
            //System.out.println("return due to bad index.");
            return returnValue; //empty string
        }
        
        addressParts = address[addressIndex].split("\\{\\{att\\}\\}");
        addressName = addressParts[0].trim();
        if (addressParts.length > 1) {
            //we had a substring.
            String[] tmp = addressParts[1].trim().split("\\{\\{val\\}\\}");
            if (tmp.length < 2) {
                log.addEntry("System","XML_SEEK_ERROR","Error in Declaration, Expecting two Values:" +addressParts[addressIndex],PBBLogging.logs.SY);
                
                System.out.println("XML_SEEK_ERROR: Error in declaration, expecting 2 values: " + addressParts[addressIndex]);
                return returnValue;
            } else {
                addressAtt = tmp[0].trim();
                addressAttVal = tmp[1].trim();
            }
            checkAtt = true;
        }
        //DEBUG BEGIN
        //System.out.println("addressName: " + addressName);
        //System.out.println("addressAtt: " + addressAtt);
        //System.out.println("addressAttVal: " + addressAttVal);
        //DEBUG END
        
        
        //We are going to look through all the child nodes until we get a match.
        NodeList children = root.getChildNodes();
        Element tempElement;
        //System.out.println("Num Kids:" + children.getLength());
        for (int i =0; i < children.getLength(); i++){
            //only want to deal with element nodes at this time.
            if (children.item(i).getNodeType() == Node.ELEMENT_NODE) {
                tempElement = (Element)children.item(i);
                //System.out.println("NodeName: " + tempElement.getNodeName());
                if (tempElement.getNodeName().compareTo(addressName) == 0){
                    //Found the next address, lets check the attribute:
                    //System.out.println("Found Address.");
                    if (checkAtt){
                        String attVal = tempElement.getAttribute(addressAtt);
                        //System.out.println("attVal=" + attVal); //DEBUG XXX
                        //System.out.println("Compare \"??\" to " + "\"" + addressAttVal + "\""); //DEBUG XXX
                        if (addressAttVal.compareTo("??") == 0){
                            //need to return the attribute value.
                            //System.out.println("Match 3."); //DEBUG XXX
                            returnValue = returnValue + valueSplit + tempElement.getAttribute(addressAtt);
                            //This is good comma behavior for us, we want the very first return value to start with a comma.
                        }else {
                            if (tempElement.getAttribute(addressAtt).compareTo(addressAttVal) == 0) {
                                //we have a match,  Descend down the tree further.
                                //System.out.println("Match 1."); //DEBUG XXX
                                returnValue = returnValue + seekValue(tempElement,address,addressIndex + 1, valueSplit);
                            }
                            //otherwise do nothing, we don't have a match.
                        }
                    } else {
                        //don't want to check the attribute, therefore, we move onward.
                        //System.out.println("Match 2."); //DEBUG XXX
                        returnValue = returnValue + seekValue(tempElement,address,addressIndex + 1, valueSplit);
                    }
                } else {
                    //we don't have a match, therefore we do nothing.
                }
            }
            //do nothing if we don't have an element node.
        }
        //System.out.println("return Value: " + returnValue); //DEBUG XXX
        return returnValue;
        
    }
    
    private void extractValues(Element root, String id) throws SAXException{
        NodeList tmpList;
        tmpList = root.getElementsByTagName(id);
        for( int i = 0; tmpList.getLength() > i; i++){
            emit(id+", " + ((Element)tmpList.item(i)).getAttribute("value"));
            nl();
        }
    }
    
    public void startDocument()
    throws SAXException {
        nl();
        nl();
        emit("START DOCUMENT");
        nl();
        //emit("<?xml version='1.0' encoding='UTF-8'?>");
    }
    
    public void endDocument() throws SAXException {
        try {
            out = new OutputStreamWriter(System.out);
            nl();
            emit("END DOCUMENT");
            nl();
            
            nl();
            out.flush();
            this.dataDump(); //Dump the Data to the Screen.
        } catch (IOException e) {
            throw new SAXException("I/O error", e);
        }
    }
    
    public void startElement(String namespaceURI, String lName, String qName, Attributes attrs) throws SAXException {
        
        //nl(); emit("ELEMENT: ");
        
        String eName = lName; // element name
        if ("".equals(eName)){
            eName = qName; // namespaceAware = false
        }
        if (eName.toUpperCase().compareTo("BIOGPS_REPORT") == 0){
            //Starting a new report, looking for the date.
            outSubDir1 = "BioGPS_Report-UNKNOWN/"; //error catch
            if (attrs != null){
                for (int i = 0; i < attrs.getLength(); i++) {
                    String aName = attrs.getLocalName(i); // Attr name
                    if ("".equals(aName)){
                        aName = attrs.getQName(i);
                    }
                    if(aName.toUpperCase().compareTo("DATECREATED") == 0){
                        outSubDir1 = "BioGPS_Report-" + attrs.getValue(i).replace(':','.') + "/";
                        
                    }
                    
                }
            } else {
                System.out.println("Error in XML: GPS Report missing Date");
            }
        } else if(eName.toUpperCase().compareTo("ORTHOLOGY_ID") == 0) {
            //start of a new gene record
            outSubDir2 = "ORTH_ID-UNKNOWN/"; //error catch
            if (attrs != null){
                for (int i = 0; i < attrs.getLength(); i++) {
                    String aName = attrs.getLocalName(i); // Attr name
                    if ("".equals(aName)){
                        aName = attrs.getQName(i);
                    }
                    if(aName.toUpperCase().compareTo("VALUE") == 0){
                        outSubDir2 = "ORTH_ID-" + attrs.getValue(i) + "/";
                        this.addData(eName.toUpperCase(),attrs.getValue(i),false); //DATA CATCH
                    }
                    
                }
            } else {
                System.out.println("Error in XML: Orthology missing number");
            }
        }else if(eName.toUpperCase().compareTo("GENEREPORT") == 0) {
            //start of a new gene record
            String temp1 = "####";
            String temp2 = "RACE";
            if (attrs != null){
                for (int i = 0; i < attrs.getLength(); i++) {
                    String aName = attrs.getLocalName(i); // Attr name
                    if ("".equals(aName)){
                        aName = attrs.getQName(i);
                    }
                    if(aName.toUpperCase().compareTo("TAXONOMY") == 0){
                        temp1 = attrs.getValue(i) + "";
                        
                    }
                    if(aName.toUpperCase().compareTo("TAXONOMYNAME") == 0){
                        temp2 = attrs.getValue(i) + "";
                    }
                    
                }
                
            } else {
                System.out.println("Error in XML: Gene Report Missing Taxonomy info.");
                
            }
            outFile = temp2 + "-" + temp1+".txt";
            //need to start a new output file stream.
            String outTempName = outDirectory+outSubDir1+outSubDir2+outFile;
            //Need to update the SpeciesCount and start a new dataset.
            speciesCount++;
            //speciesData.add(new Hashtable<String, Vector>());
            try {
                nl();
                out.flush();
            } catch (IOException e) {
                throw new SAXException("I/O error", e);
            }
            System.out.println("Setting new output Directory: "+ outTempName);
            out = this.setOutFile(outTempName);
            if (out == null) {
                out = new OutputStreamWriter(System.out);
            }
            
        }else{
            //normal processing
            //Going to try and get sub data tabbed over in output.
            nl(); //emit("ELEMENT: ");
            emit(spacer + eName);
            if (attrs != null) {
                for (int i = 0; i < attrs.getLength(); i++) {
                    String aName = attrs.getLocalName(i); // Attr name
                    if ("".equals(aName)){
                        aName = attrs.getQName(i);
                    }
                    //nl();
                    //emit("   ATTR: ");
                    if(aName.toUpperCase().compareTo("VALUE") != 0){
                        
                        emit(", "+ aName+"=");
                        emit(attrs.getValue(i) + " ");
                        this.addData(aName.toUpperCase(),attrs.getValue(i),false); //DATA CATCH
                    } else {
                        //emit("\t\"");
                        emit(", "+ attrs.getValue(i) + " ");
                        this.addData(eName.toUpperCase(),attrs.getValue(i),false); //DATA CATCH
                        //emit("\"");
                    }
                }
                //since we are opening an element, we add a space.
                spacer = spacer + "   ";
            }
        }
        //if (attrs.getLength() > 0) nl();
        //emit(">");
    }
    
    public void addData(String arg, String val, boolean overwrite) {
        //Will add a new value to the argument arg.  If overwrite is true, then old values
        //will be removed, else the new value will be added to the data vector.
        
        //Fist get the proper Species...
        Hashtable<String, Vector> dataHash;
        if (speciesCount < speciesData.size() && speciesCount >= 0){
            dataHash = speciesData.get(speciesCount);
            if (dataHash == null){
                dataHash = new Hashtable<String, Vector>();
            }
        }else {
            dataHash = new Hashtable<String, Vector>();
            speciesData.add(dataHash);
            if (speciesCount != speciesData.size() - 1){
                //we have a serious problem
                System.out.println("Big Error while Adding data, integer not synced.");
            }
        }
        Vector<String> valList;
        
        if (overwrite) {
            //overwrite the pervious value, if it existed.
            valList = new Vector<String>();
            valList.add(val);
            dataHash.put(arg,valList);
        } else {
            if( dataHash.containsKey(arg)){
                //already exists.
                valList = dataHash.get(arg);
                //does the value already exist?
                if (!valList.contains(val)) {
                    
                    valList.add(val);
                }
            } else {
                valList = new Vector<String>();
                valList.add(val);
                dataHash.put(arg,valList);
            }
        }
        
        return;
    }
    
    
//Depricated Method
    public void dataDump() throws SAXException {
        nl();
        emit("Dumping all data from structure...");
        nl();
        try {
            nl();
            out.flush();
        } catch (IOException e) {
            throw new SAXException("I/O error", e);
        }
        System.out.println("Setting new output for dump.");
        out = this.setOutFile("c:/temp/datadump.txt");
        if (out == null) {
            out = new OutputStreamWriter(System.out);
        }
        Hashtable<String, Vector> dataHash;
        
        for (int v = 0; v < speciesData.size(); v++){
            dataHash = speciesData.get(v);
            
            Enumeration<String> theKeys;
            theKeys = dataHash.keys();
            String key;
            Vector<String> valList;
            String val;
            nl();
            emit("# **** Species: " + v);
            nl();
            
            while (theKeys.hasMoreElements()) {
                key = theKeys.nextElement();
                valList = dataHash.get(key);
                nl();
                emit(key + ", ");
                if (valList != null){
                    for (int i=0; i< valList.size(); i++){
                        val = valList.get(i);
                        if (i == valList.size() - 1) {
                            //Last value, no Comma.
                            emit(val);
                        }else {
                            emit(val +", ");
                        }
                        
                    }
                }
            }
        }
        nl();
        System.out.println("Finished data dump");
        nl();
        
    }
    
    public void endElement(String namespaceURI,
            String sName, // simple name
            String qName  // qualified name
            )
            throws SAXException {
        //nl();
        //ending an element lets us remove the spaces (if any).
        if (spacer.length() <= 3){
            spacer = "";
        }else {
            spacer = spacer.substring(3);
        }
        //emit("END_ELM: ");
        //emit("</"+qName+">");
    }
    
    public void characters(char buf[], int offset, int len)
    throws SAXException {
        String s = new String(buf, offset, len).trim();
        
        if(s.compareTo("") != 0) {
            nl(); emit("CHARS: |");
            
            
            emit(s);
            emit("|");
        }
    }
    
    
//Borrowed from the Java tutorial on XML.
    private void emit(String s) throws SAXException {
        try {
            out.write(s);
            out.flush();
        } catch (IOException e) {
            throw new SAXException("I/O error", e);
        }
    }
    
// Start a new line
    private void nl() throws SAXException {
        String lineEnd =  System.getProperty("line.separator");
        try {
            out.write(lineEnd);
        } catch (IOException e) {
            throw new SAXException("I/O error", e);
        }
    }
    
    
}
